Benchmark & algorithms setup

Stop conditions:

Maximal number of function evaluations reached.

Restart triggers:

- Standard deviation below tolerance in all coordinates.
- Condition number of covariance matrix exceeds 1e14.
- Covariance matrix is not numerically positive definite.
- Addition of 0.1 times sigma does not change mean value.
- Addition of 0.2 times sigma in any coordinate does not change mean value.

Results

library(cecb)
library(tidyverse)

knitr::opts_chunk$set(fig.width=12, fig.height=10) 

"%+%" <- function(str1, str2) {
  paste0(str1, str2)
}
prefix = "../data/"

csa = list(
    basic = prefix %+% "csa/ipop-cma-es-csa-basic",
    rot = prefix %+% "csa/ipop-cma-es-csa-rot",
    shift = prefix %+% "csa/ipop-cma-es-csa-shift",
    bias = prefix %+% "csa/ipop-cma-es-csa-bias",
    bias_shift_rot = prefix %+% "csa/ipop-cma-es-csa-bias-shift-rot",
    shift_rot = prefix %+% "csa/ipop-cma-es-csa-shift-rot",
    bias_rot = prefix %+% "csa/ipop-cma-es-csa-bias-rot",
    bias_shift = prefix %+% "csa/ipop-cma-es-csa-bias-shift"
)
ppmf = list(
    basic = prefix %+% "ppmf/ipop-cma-es-ppmf-basic",
    rot = prefix %+% "ppmf/ipop-cma-es-ppmf-rot",
    shift = prefix %+% "ppmf/ipop-cma-es-ppmf-shift",
    bias = prefix %+% "ppmf/ipop-cma-es-ppmf-bias",
    bias_shift_rot = prefix %+% "ppmf/ipop-cma-es-ppmf-bias-shift-rot",
    shift_rot = prefix %+% "ppmf/ipop-cma-es-ppmf-shift-rot",
    bias_rot = prefix %+% "ppmf/ipop-cma-es-ppmf-bias-rot",
    bias_shift = prefix %+% "ppmf/ipop-cma-es-ppmf-bias-shift"
)
config = list(dim = 10, problems = 1:10, repetitions = 30)
config2 = list(dim = 20, problems = 1:10, repetitions = 30)

1. Basic

ECDF curves N = 10

cecb::cec_class_grid(c(csa$basic, ppmf$basic), 10, 21, 30)

cecb::cec_problem_grid(c(csa$basic, ppmf$basic), 21, config)

CEC’s tables N = 10

cecb::get_resultTable(csa$basic, 1:10, 10, caption = "CSA (basic)")
CSA (basic)
Function Best Worst Median Mean Std
1 1.381970e-08 5.565171e+09 3.200969e-04 6.269933e+08 1.627710e+09
2 2.715832e+03 3.306615e+03 2.715832e+03 2.792275e+03 1.651358e+02
3 1.263507e+01 7.068753e+02 1.263507e+01 9.510139e+01 1.886028e+02
4 8.799318e-01 6.203364e+04 8.801303e-01 4.501460e+03 1.656085e+04
5 7.587359e+02 1.237040e+09 7.587360e+02 8.933720e+07 3.303472e+08
6 4.760753e+02 6.493528e+02 4.762086e+02 5.176548e+02 7.093617e+01
7 1.846050e+01 3.213260e+09 2.007417e+01 2.297765e+08 8.587060e+08
8 1.741605e+03 2.004195e+03 1.752222e+03 1.775990e+03 7.230962e+01
9 5.218806e-07 1.144095e+03 6.227551e-07 1.574901e+02 3.343981e+02
10 4.799559e+01 1.180260e+03 4.804775e+01 1.445138e+02 3.016664e+02
cecb::get_resultTable(ppmf$basic, 1:10, 10, caption = "PPMF (basic)")
PPMF (basic)
Function Best Worst Median Mean Std
1 4.526549e-08 9.234456e+09 1.237334e-02 7.754739e+08 2.467175e+09
2 1.253918e+03 2.955554e+03 1.625398e+03 1.701811e+03 4.436810e+02
3 3.052620e+01 7.374674e+01 5.698455e+01 5.608301e+01 1.527645e+01
4 2.253222e+00 5.325964e+00 3.094592e+00 3.544897e+00 1.070668e+00
5 2.222612e+02 1.332106e+07 2.225184e+02 1.473110e+06 3.629690e+06
6 6.738882e-01 9.789427e+01 9.789427e+01 7.713761e+01 4.124658e+01
7 1.526430e+00 4.452000e+06 5.875382e+01 6.905335e+05 1.429682e+06
8 5.238032e-13 2.285284e+03 1.248677e+02 4.822427e+02 7.719461e+02
9 6.016171e-07 1.110196e+03 7.176749e-07 1.462584e+02 3.351837e+02
10 4.892959e+01 7.805625e+01 4.958008e+01 5.164326e+01 7.610996e+00

ECDF curves N = 20

cecb::cec_class_grid(c(csa$basic, ppmf$basic), 20, 21, 30)

cecb::cec_problem_grid(c(csa$basic, ppmf$basic), 21, config2)

CEC’s tables N = 20

cecb::get_resultTable(csa$basic, 1:10, 20, caption = "CSA (basic)")
CSA (basic)
Function Best Worst Median Mean Std
1 1.256053e-07 1.436069e+10 1.989033e-07 1.377472e+09 3.924211e+09
2 3.803479e+03 4.951710e+03 3.803479e+03 3.942585e+03 3.275980e+02
3 2.851593e+01 1.239561e+03 2.851593e+01 1.783304e+02 3.340868e+02
4 1.817663e+00 5.889763e+04 2.119466e+00 4.249638e+03 1.572944e+04
5 8.021378e+02 6.814873e+07 8.021378e+02 7.543758e+06 1.902601e+07
6 2.042990e+00 2.081009e+03 2.208723e+00 2.674111e+02 5.781545e+02
7 7.320058e+02 1.181598e+08 7.989452e+02 9.192543e+06 3.143939e+07
8 2.714334e+03 4.793458e+03 2.714334e+03 3.049387e+03 7.020230e+02
9 6.392806e-07 1.154218e+03 9.101145e-07 1.373803e+02 3.340682e+02
10 4.878192e+01 9.266191e+02 4.881564e+01 1.235826e+02 2.339151e+02
cecb::get_resultTable(ppmf$basic, 1:10, 20, caption = "PPMF (basic)")
PPMF (basic)
Function Best Worst Median Mean Std
1 1.124458e-07 3.675474e+09 1.257015e-02 2.711061e+08 9.803511e+08
2 4.350615e+02 4.624970e+03 2.532815e+03 2.558411e+03 8.225195e+02
3 2.323253e+01 1.636152e+02 1.306078e+02 1.271013e+02 3.197222e+01
4 8.110247e+00 1.161189e+01 8.352910e+00 9.071896e+00 1.359803e+00
5 1.037451e+03 1.152268e+07 1.043481e+03 9.184064e+05 3.059267e+06
6 2.059973e+02 8.994298e+02 2.746626e+02 3.722916e+02 2.402816e+02
7 4.414605e+02 1.492077e+07 6.165430e+02 1.546500e+06 4.136990e+06
8 1.129104e+01 5.821674e+03 1.129104e+01 1.031071e+03 1.935579e+03
9 7.252626e-07 1.837796e+03 7.252626e-07 2.084292e+02 5.212980e+02
10 4.879225e+01 7.715242e+01 4.879225e+01 5.158342e+01 7.435791e+00

2. Rot

ECDF curves N = 10

cecb::cec_class_grid(c(csa$rot, ppmf$rot), 10, 21, 30)

cecb::cec_problem_grid(c(csa$rot, ppmf$rot), 21, config)

CEC’s tables N = 10

cecb::get_resultTable(csa$rot, 1:10, 10, caption = "CSA (rot)")
CSA (rot)
Function Best Worst Median Mean Std
1 3.291628e-08 4.270728e+09 2.250660e-03 5.441650e+08 1.271176e+09
2 1.492080e+03 2.334146e+03 1.492080e+03 1.610302e+03 2.465251e+02
3 1.441855e+01 6.751787e+02 1.441855e+01 9.205052e+01 1.793975e+02
4 6.057982e-01 1.435883e+04 6.057988e-01 1.273526e+03 3.876453e+03
5 2.712272e+02 1.028810e+07 2.713273e+02 8.672508e+05 2.723878e+06
6 1.173368e+00 8.303983e+02 1.705386e+00 1.084915e+02 2.316881e+02
7 3.755383e+01 5.015480e+09 5.303967e+01 3.582962e+08 1.340430e+09
8 1.327306e+03 1.952857e+03 1.327306e+03 1.399494e+03 1.834003e+02
9 5.570145e-07 5.856929e+02 6.900576e-07 7.414447e+01 1.658055e+02
10 7.192467e+01 2.852177e+02 7.202770e+01 1.056930e+02 7.673065e+01
cecb::get_resultTable(ppmf$rot, 1:10, 10, caption = "PPMF (rot)")
PPMF (rot)
Function Best Worst Median Mean Std
1 3.538610e-08 6.951559e+09 2.194341e-01 5.460441e+08 1.851596e+09
2 4.306353e+02 1.959936e+03 1.437636e+03 1.217029e+03 5.996686e+02
3 3.942626e+01 6.995433e+01 4.729791e+01 5.110510e+01 7.710821e+00
4 2.591483e+00 5.765637e+00 2.591483e+00 3.341175e+00 1.090444e+00
5 6.910066e+02 1.711381e+07 7.111487e+02 1.537553e+06 4.548023e+06
6 5.941119e+00 3.004036e+02 1.092354e+02 1.172899e+02 9.439337e+01
7 1.186603e+02 3.094054e+06 1.186647e+02 2.952324e+05 8.232478e+05
8 9.293307e-13 1.770004e+03 2.420919e-05 4.416629e+02 7.222227e+02
9 3.749865e-07 9.210357e+02 1.006669e-06 9.980013e+01 2.522753e+02
10 7.214410e+01 7.732678e+01 7.220499e+01 7.308074e+01 1.510819e+00

ECDF curves N = 20

cecb::cec_class_grid(c(csa$rot, ppmf$rot), 20, 21, 30)

cecb::cec_problem_grid(c(csa$rot, ppmf$rot), 21, config2)

CEC’s tables N = 20

cecb::get_resultTable(csa$rot, 1:10, 20, caption = "CSA (rot)")
CSA (rot)
Function Best Worst Median Mean Std
1 1.197396e-07 1.972448e+10 1.214671e-06 2.353251e+09 5.828524e+09
2 2.760793e+03 5.304385e+03 2.760793e+03 3.282963e+03 9.749652e+02
3 2.364653e+01 2.232922e+03 2.364653e+01 3.030187e+02 6.173612e+02
4 2.626448e+00 1.400797e+05 2.647234e+00 1.004682e+04 3.742628e+04
5 6.679094e+02 5.642817e+07 6.679094e+02 5.518158e+06 1.542984e+07
6 2.007702e+00 2.243153e+03 2.110144e+00 3.244624e+02 6.683698e+02
7 4.450762e+02 6.932164e+07 4.470159e+02 5.892523e+06 1.840885e+07
8 3.645783e+03 5.058944e+03 3.656941e+03 3.877932e+03 4.771066e+02
9 5.337911e-07 8.241308e+02 8.187200e-07 1.149554e+02 2.581382e+02
10 6.234187e+01 2.036346e+03 6.241046e+01 2.195378e+02 5.244255e+02
cecb::get_resultTable(ppmf$rot, 1:10, 20, caption = "PPMF (rot)")
PPMF (rot)
Function Best Worst Median Mean Std
1 1.282664e-07 1.171851e+10 6.289459e+01 8.503153e+08 3.128460e+09
2 5.733009e+02 4.061164e+03 2.720853e+03 2.662116e+03 6.994627e+02
3 2.253646e+01 1.309147e+02 1.297369e+02 1.203097e+02 2.843293e+01
4 7.385819e+00 1.242010e+01 8.936705e+00 9.334797e+00 1.648877e+00
5 1.446628e+03 5.588068e+07 1.446975e+03 4.118005e+06 1.490438e+07
6 1.499553e+02 2.956326e+02 1.987124e+02 2.162766e+02 5.524179e+01
7 3.727987e+02 2.179846e+07 4.520794e+02 1.671854e+06 5.799003e+06
8 2.121893e+03 5.432717e+03 2.121893e+03 2.710021e+03 1.207235e+03
9 6.961068e-07 1.992283e+03 1.157334e-06 2.197462e+02 5.626468e+02
10 6.235405e+01 2.147485e+02 6.241037e+01 7.790060e+01 4.039846e+01

3. Shift

ECDF curves N = 10

cecb::cec_class_grid(c(csa$shift, ppmf$shift), 10, 21, 30)

cecb::cec_problem_grid(c(csa$shift, ppmf$shift), 21, config)

CEC’s tables N = 10

cecb::get_resultTable(csa$shift, 1:10, 10, caption = "CSA (shift)")
CSA (shift)
Function Best Worst Median Mean Std
1 2.526855e-08 3.854679e+09 1.081869e-04 5.315445e+08 1.290996e+09
2 9.499729e+02 2.280037e+03 9.499729e+02 1.085011e+03 3.624137e+02
3 1.296279e+01 1.485941e+03 1.296279e+01 1.578600e+02 3.927642e+02
4 8.946582e-01 2.537162e+05 1.080651e+00 1.847241e+04 6.772012e+04
5 7.220152e+02 1.957572e+09 7.220154e+02 1.407798e+08 5.229169e+08
6 5.176949e+02 6.467574e+02 5.186769e+02 5.429984e+02 4.266582e+01
7 2.228318e+02 1.276313e+07 2.434813e+02 1.949950e+06 3.971860e+06
8 1.000000e+02 3.280595e+03 1.000000e+02 4.244082e+02 8.688850e+02
9 3.280391e+02 1.717819e+03 3.280391e+02 4.492744e+02 3.694334e+02
10 4.000000e+02 4.229524e+03 4.000000e+02 8.546243e+02 1.067593e+03
cecb::get_resultTable(ppmf$shift, 1:10, 10, caption = "PPMF (shift)")
PPMF (shift)
Function Best Worst Median Mean Std
1 2.534186e-08 2.453955e+10 6.553811e-03 2.144216e+09 6.572666e+09
2 1.254582e+03 2.417254e+03 1.254582e+03 1.546888e+03 4.817112e+02
3 3.284677e+01 6.834951e+01 5.493681e+01 5.597986e+01 1.150531e+01
4 2.450217e+00 4.931657e+00 2.999026e+00 3.239135e+00 8.973915e-01
5 6.461693e+02 4.478067e+07 6.462548e+02 3.845844e+06 1.188652e+07
6 1.185706e+02 4.498320e+02 2.527457e+02 2.592226e+02 1.348198e+02
7 1.220629e+02 3.052262e+09 1.521296e+02 2.182513e+08 8.156846e+08
8 1.000000e+02 2.810087e+03 1.000000e+02 5.076926e+02 8.384198e+02
9 3.280391e+02 5.111418e+02 3.670626e+02 3.873709e+02 5.308716e+01
10 4.000000e+02 2.318159e+03 4.000000e+02 6.792952e+02 5.714756e+02

ECDF curves N = 20

cecb::cec_class_grid(c(csa$shift, ppmf$shift), 20, 21, 30)

cecb::cec_problem_grid(c(csa$shift, ppmf$shift), 21, config2)

CEC’s tables N = 20

cecb::get_resultTable(csa$shift, 1:10, 20, caption = "CSA (shift)")
CSA (shift)
Function Best Worst Median Mean Std
1 7.271646e-08 2.857270e+10 1.339719e-07 3.232032e+09 8.120120e+09
2 1.334005e+03 5.705092e+03 1.334005e+03 2.514576e+03 1.796193e+03
3 2.418078e+01 8.386204e+02 2.418078e+01 1.519295e+02 2.461261e+02
4 1.811143e+00 1.332874e+06 1.898432e+00 9.563394e+04 3.561056e+05
5 5.687699e+02 7.371413e+07 5.687700e+02 1.133736e+07 2.657573e+07
6 2.395442e+02 1.657872e+03 2.395689e+02 5.386126e+02 5.474494e+02
7 5.724559e+02 7.897723e+08 8.721479e+02 6.098752e+07 2.100433e+08
8 4.098671e+03 5.842267e+03 4.098671e+03 4.399521e+03 6.113525e+02
9 3.928493e+02 1.386595e+03 3.928493e+02 5.536478e+02 2.999575e+02
10 4.857058e+02 1.592724e+03 4.857284e+02 6.672628e+02 3.980828e+02
cecb::get_resultTable(ppmf$shift, 1:10, 20, caption = "PPMF (shift)")
PPMF (shift)
Function Best Worst Median Mean Std
1 1.057234e-07 1.398530e+10 39.59656 1.026911e+09 3.731069e+09
2 2.274955e+03 5.155636e+03 4300.51145 4.427477e+03 7.273574e+02
3 2.261837e+01 1.647927e+02 102.36926 1.153381e+02 3.534427e+01
4 6.727819e+00 1.099325e+01 8.74878 8.811822e+00 1.433594e+00
5 1.189102e+03 5.393156e+07 1604.38332 4.853574e+06 1.435204e+07
6 3.795907e+02 1.260461e+03 922.02584 8.481167e+02 2.516111e+02
7 5.643498e+02 6.213818e+06 564.81947 4.817957e+05 1.653048e+06
8 1.000000e+02 6.044144e+03 100.00000 9.280356e+02 1.912370e+03
9 3.967286e+02 1.721342e+03 411.60179 6.641334e+02 4.244731e+02
10 4.857117e+02 5.109860e+02 485.71170 4.876818e+02 6.731929e+00

4. Bias

ECDF curves N = 10

cecb::cec_class_grid(c(csa$bias, ppmf$bias), 10, 21, 30)

cecb::cec_problem_grid(c(csa$bias, ppmf$bias), 21, config)

CEC’s tables N = 10

cecb::get_resultTable(csa$bias, 1:10, 10, caption = "CSA (bias)")
CSA (bias)
Function Best Worst Median Mean Std
1 3.988036e-08 2.157485e+10 9.489354e-06 1.940291e+09 5.799666e+09
2 6.591886e+02 2.390713e+03 6.591886e+02 1.042529e+03 6.454640e+02
3 1.263507e+01 7.417596e+02 1.263507e+01 9.729713e+01 1.974213e+02
4 4.868443e-01 3.066726e+03 4.868443e-01 2.377414e+02 8.167479e+02
5 9.290403e+02 2.077320e+07 9.290404e+02 1.587631e+06 5.526559e+06
6 3.576970e+02 7.217731e+02 3.577493e+02 4.167949e+02 1.172628e+02
7 3.414936e+01 1.360581e+08 4.211715e+01 9.723208e+06 3.636170e+07
8 1.159368e+03 1.593345e+03 1.159368e+03 1.234316e+03 1.589917e+02
9 5.738707e-07 3.079668e+02 1.013336e-06 4.104929e+01 8.868482e+01
10 4.799559e+01 4.548818e+02 4.801011e+01 8.348626e+01 1.082281e+02
cecb::get_resultTable(ppmf$bias, 1:10, 10, caption = "PPMF (bias)")
PPMF (bias)
Function Best Worst Median Mean Std
1 2.600997e-08 8.763281e+09 5.332626e-05 6.470525e+08 2.337070e+09
2 5.548191e+02 2.264993e+03 1.506039e+03 1.370307e+03 5.028775e+02
3 3.639134e+01 5.467233e+01 5.137314e+01 4.897954e+01 7.021258e+00
4 2.277531e+00 5.373099e+00 3.056537e+00 3.189242e+00 9.343251e-01
5 2.223226e+02 1.691341e+07 2.237840e+02 1.350856e+06 4.497769e+06
6 2.333065e+01 4.448363e+02 8.458234e+01 1.273700e+02 1.079669e+02
7 4.994350e+02 7.435401e+04 5.992492e+02 1.140780e+04 2.268470e+04
8 9.094947e-13 2.451228e+03 1.119581e+02 4.863120e+02 8.183874e+02
9 4.395292e-07 8.647112e+02 7.801864e-07 9.973639e+01 2.395103e+02
10 7.723921e+01 8.797692e+01 7.862770e+01 7.997452e+01 3.108196e+00

ECDF curves N = 20

cecb::cec_class_grid(c(csa$bias, ppmf$bias), 20, 21, 30)

cecb::cec_problem_grid(c(csa$bias, ppmf$bias), 21, config2)

CEC’s tables N = 20

cecb::get_resultTable(csa$bias, 1:10, 20, caption = "CSA (bias)")
CSA (bias)
Function Best Worst Median Mean Std
1 6.100890e-08 1.256260e+10 2.362681e-07 1.504527e+09 3.697744e+09
2 3.384225e+03 5.205939e+03 3.384225e+03 3.704318e+03 6.497490e+02
3 4.974795e+00 1.043228e+03 4.974795e+00 1.735282e+02 3.380019e+02
4 1.919038e+00 4.706767e+04 1.964129e+00 3.377534e+03 1.257496e+04
5 7.978946e+02 1.651964e+07 7.978946e+02 3.087709e+06 6.218631e+06
6 1.903105e+00 1.820802e+03 2.189857e+00 2.696524e+02 5.664225e+02
7 5.166332e+02 7.387466e+07 6.572395e+02 6.067723e+06 1.965043e+07
8 9.094947e-13 6.297511e+03 3.183231e-12 1.091040e+03 2.101147e+03
9 7.339945e-07 1.449690e+03 9.136534e-07 1.978308e+02 4.525709e+02
10 4.878667e+01 1.204472e+03 4.891135e+01 1.476663e+02 3.077114e+02
cecb::get_resultTable(ppmf$bias, 1:10, 20, caption = "PPMF (bias)")
PPMF (bias)
Function Best Worst Median Mean Std
1 1.217100e-07 1.188383e+10 6.479025e-01 8.685658e+08 3.171236e+09
2 2.185992e-01 4.741702e+03 2.142340e+03 2.745490e+03 1.391027e+03
3 2.232029e+01 1.466569e+02 1.234593e+02 1.231830e+02 3.167088e+01
4 8.704378e+00 1.316002e+01 8.928642e+00 9.390738e+00 1.155744e+00
5 8.064785e+02 8.257216e+06 8.065628e+02 1.047505e+06 2.461934e+06
6 1.587376e+00 8.373767e+02 1.801657e+02 2.739186e+02 2.314481e+02
7 1.115584e+03 1.688510e+06 1.415237e+03 2.907013e+05 5.419558e+05
8 2.557592e+03 5.222330e+03 2.713232e+03 2.983658e+03 7.152224e+02
9 6.581081e-07 1.222955e+03 1.259285e-06 1.241266e+02 3.317543e+02
10 4.878599e+01 6.358557e+01 5.099172e+01 5.106569e+01 3.807078e+00

5. Bias-shift-rot

ECDF curves N = 10

cecb::cec_class_grid(c(csa$bias_shift_rot, ppmf$bias_shift_rot), 10, 21, 30)

cecb::cec_problem_grid(c(csa$bias_shift_rot, ppmf$bias_shift_rot), 21, config)

CEC’s tables N = 10

cecb::get_resultTable(csa$bias_shift_rot, 1:10, 10, caption = "CSA (bias x shift x rot)")
CSA (bias x shift x rot)
Function Best Worst Median Mean Std
1 1.878216e-08 3.267344e+10 1.936252e-04 3.017906e+09 8.774831e+09
2 2.503784e+03 2.703859e+03 2.503784e+03 2.533087e+03 6.297542e+01
3 1.442334e+01 8.444371e+02 1.442334e+01 1.039160e+02 2.210380e+02
4 2.628805e-01 3.601702e+06 1.005825e+00 3.018885e+05 9.642014e+05
5 1.714270e+01 4.299388e+07 1.714277e+01 3.545077e+06 1.141353e+07
6 1.209039e+02 2.685284e+03 1.209603e+02 4.045387e+02 6.846469e+02
7 3.108539e+02 2.029269e+06 3.129592e+02 2.983111e+05 6.298108e+05
8 1.000000e+02 1.875163e+03 1.000000e+02 2.623945e+02 4.744942e+02
9 2.000000e+02 9.202337e+02 2.000000e+02 2.819787e+02 2.025073e+02
10 4.433300e+02 2.309982e+03 4.433339e+02 6.199665e+02 4.978345e+02
cecb::get_resultTable(ppmf$bias_shift_rot, 1:10, 10, caption = "PPMF (bias x shift x rot)")
PPMF (bias x shift x rot)
Function Best Worst Median Mean Std
1 5.089846e-08 1.320216e+10 1.816404 1.163346e+09 3.541659e+09
2 1.329538e+03 1.686187e+03 1606.467488 1.587868e+03 1.303366e+02
3 4.114522e+01 5.983890e+01 59.838897 5.385413e+01 7.601810e+00
4 2.140687e+00 8.532456e+00 2.757814 3.333276e+00 1.590034e+00
5 5.742098e+02 3.146300e+07 725.485593 2.371815e+06 8.376970e+06
6 5.061811e+01 4.439373e+02 219.581850 2.383743e+02 1.254092e+02
7 1.192477e+02 9.006818e+06 1292.335354 7.142417e+05 2.391226e+06
8 1.000000e+02 2.795206e+03 100.000000 4.731118e+02 8.217573e+02
9 2.311340e+02 4.810149e+02 302.005500 3.160133e+02 9.210612e+01
10 3.995850e+02 6.872566e+02 399.584968 4.237000e+02 7.687708e+01

6. Shift-rot

ECDF curves N = 10

#cecb::cec_class_grid(c(csa$shift_rot, ppmf$shift_rot), 10, 21, 30)
#cecb::cec_problem_grid(c(csa$shift_rot, ppmf$shift_rot), 21, config)

CEC’s tables N = 10

#cecb::get_resultTable(csa$shift_rot, 1:10, 10, caption = "CSA (shift x rot)")
#cecb::get_resultTable(ppmf$shift_rot, 1:10, 10, caption = "PPMF (shift x rot)")

7. Bias-shift

ECDF curves N = 10

cecb::cec_class_grid(c(csa$bias_shift, ppmf$bias_shift), 10, 21, 30)

cecb::cec_problem_grid(c(csa$bias_shift, ppmf$bias_shift), 21, config)

CEC’s tables N = 10

cecb::get_resultTable(csa$bias_shift, 1:10, 10, caption = "CSA (bias x shift)")
CSA (bias x shift)
Function Best Worst Median Mean Std
1 4.131093e-08 1.664492e+10 3.369410e-04 1.829450e+09 4.701525e+09
2 1.101039e+03 1.983378e+03 1.101039e+03 1.233498e+03 2.977865e+02
3 1.202070e+01 1.186805e+03 1.202145e+01 1.369468e+02 3.138284e+02
4 1.138354e+00 1.238040e+05 1.138354e+00 8.941890e+03 3.306153e+04
5 3.876389e+02 6.865880e+07 3.876395e+02 6.099723e+06 1.833671e+07
6 3.487432e+02 1.003378e+03 3.488138e+02 4.408434e+02 1.867279e+02
7 2.531842e+02 8.991159e+05 2.590549e+02 1.770676e+05 3.122170e+05
8 1.305727e+03 2.375342e+03 1.305728e+03 1.552043e+03 4.280307e+02
9 3.326047e+02 1.437185e+03 3.326047e+02 4.432995e+02 2.960228e+02
10 4.000000e+02 1.232669e+03 4.000000e+02 5.193613e+02 2.416465e+02
cecb::get_resultTable(ppmf$bias_shift, 1:10, 10, caption = "PPMF (bias x shift)")
PPMF (bias x shift)
Function Best Worst Median Mean Std
1 5.417454e-08 1.890003e+10 7.906463e-05 2.239569e+09 5.457321e+09
2 1.146692e+03 2.140574e+03 1.572035e+03 1.698711e+03 3.117118e+02
3 3.884024e+01 6.827778e+01 5.267662e+01 5.449079e+01 1.014933e+01
4 2.187336e+00 7.310192e+00 3.302185e+00 3.779319e+00 1.327094e+00
5 1.651980e+02 6.684228e+06 1.652079e+02 5.884073e+05 1.771812e+06
6 2.036382e+02 5.932203e+02 2.036382e+02 2.922550e+02 1.335005e+02
7 1.944857e+01 7.642726e+06 4.134173e+01 1.087390e+06 2.230793e+06
8 1.000000e+02 7.809233e+02 1.000000e+02 1.734911e+02 1.882576e+02
9 3.277693e+02 4.785712e+02 3.684807e+02 3.859714e+02 3.776618e+01
10 4.000000e+02 1.179379e+03 4.000000e+02 5.010731e+02 2.189946e+02

8. Bias-rot

ECDF curves N = 10

#cecb::cec_class_grid(c(csa$bias_rot, ppmf$bias_shift), 10, 21, 30)
#cecb::cec_problem_grid(c(csa$bias_rot, ppmf$bias_shift), 21, config)

CEC’s tables N = 10

#cecb::get_resultTable(csa$bias_rot, 1:10, 10, caption = "CSA (bias x rot)")
#cecb::get_resultTable(ppmf$bias_rot, 1:10, 10, caption = "PPMF (bias x rot)")

ECDF curves N = 20

#cecb::cec_class_grid(c(csa$bias_rot, ppmf$bias_shift), 20, 21, 30)
#cecb::cec_problem_grid(c(csa$bias_rot, ppmf$bias_shift), 21, config2)

CEC’s tables N = 20

#cecb::get_resultTable(csa$bias_rot, 1:10, 20, caption = "CSA (bias x rot)")
#cecb::get_resultTable(ppmf$bias_rot, 1:10, 20, caption = "PPMF (bias x rot)")